Home / Deep Learning

Tensorflow

Image Augmentations with TensorFlow

In this article, we go through a number of TensorFlow tools for image augmentations. For the sake of demonstrations, we use sklearn image dataset.

In [1]:
from sklearn.datasets import load_sample_images
from matplotlib.font_manager import FontProperties
import matplotlib.pyplot as plt
Images = load_sample_images()
Names = [x.split("\\")[-1].replace('.jpg','').title() for x in Images['filenames']]
Images = Images['images']

def ImShow(Images, Names, Title = 'Images'):
    fig, ax = plt.subplots(1, 2 , figsize = (17, 6))
    ax = ax.ravel()
    font = FontProperties()
    font.set_weight('bold')
    for i in range(2):
        _ = ax[i].imshow(Images[i])
        _ = ax[i].axis('tight')
        _ = ax[i].set_aspect(aspect=1.0)
        _ = ax[i].axis('off')
        _ = ax[i].set_title(Names[i], fontproperties=font, fontsize = 16)
    _ = fig.subplots_adjust(wspace= 0.01)
    if Title:
        _ = fig.suptitle(Title, fontproperties=font, fontsize = 18)
    return fig, ax
# Image Show
_,_ = ImShow(Images, Names, Title = 'Original Images')
Adjustments

Adjusting Brightness

In RGB color space, brightness can be regarded as can be thought of as the arithmetic means of the red, green, and blue color coordinates. $$\text{Brightness} = \frac{R+G+G}{3}$$ the brightness of RGB or Grayscale images can be adjusted using tf.image.adjust_brightness.

In [2]:
import tensorflow as tf
Augmented_Images = Images.copy()
for i in range(2):
    Augmented_Images[i] = tf.image.adjust_brightness(Images[i], delta=0.25)
# Image Show
_,_ = ImShow(Augmented_Images, Names, Title = 'Increased Brightness of the Orignal Images by %25')
#
Augmented_Images = Images.copy()
for i in range(2):
    Augmented_Images[i] = tf.image.adjust_brightness(Images[i], delta= - 0.25)
_,_ = ImShow(Augmented_Images, Names, Title = 'Reduced Brightness of the Orignal Images by %25')

Adjusting Contrast

The contrast of RGB or grayscale images can be adjusted using tf.image.adjust_contrast.

In [3]:
Augmented_Images = Images.copy()
for i in range(2):
    Augmented_Images[i] = tf.image.adjust_contrast(Images[i], contrast_factor= 1.5)
# Image Show
ImShow(Augmented_Images, Names, Title = 'Increased Brightness of the Orignal Images by 1.5 Units')
#
Augmented_Images = Images.copy()
for i in range(2):
    Augmented_Images[i] = tf.image.adjust_contrast(Images[i], contrast_factor= -0.5)
_,_ = ImShow(Augmented_Images, Names, Title = 'Reduced Brightness of the Orignal Images 0.5 Units')

Adjusting Gamma

We can also perform Gamma Correction f.image.adjust_gamma.

In [4]:
Augmented_Images = Images.copy()
for i in range(2):
    Augmented_Images[i] = tf.image.adjust_gamma(Images[i], gamma= 4, gain = 1)
# Image Show
_,_ = ImShow(Augmented_Images, Names, Title = 'Increased Gamma and Gains')

Adjusting Hue

Adjust hue of RGB images using tf.image.adjust_hue. Delta here is a real number between -1 and 1.

In [5]:
Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.adjust_hue(Images[0], delta = .5)
Aug_Names[0]+=' (Increased Hue)'
Augmented_Images[1] = tf.image.adjust_hue(Images[1], delta = -.5)
Aug_Names[1]+=' (Decreased Hue)'
# Image Show
_,_ = ImShow(Augmented_Images, Aug_Names, Title = False)

Adjusting JPEG Quality

We also can adjust JPEG encoding quality of an image using tf.image.adjust_jpeg_quality.

In [6]:
Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.adjust_jpeg_quality(Images[0], jpeg_quality = 50)
Aug_Names[0]+=' (Decreased Quality to %50)'
Augmented_Images[1] = tf.image.adjust_jpeg_quality(Images[1], jpeg_quality = 5)
Aug_Names[1]+=' (Decreased Quality to %5)'
# Image Show
_,_ = ImShow(Augmented_Images, Aug_Names, Title = False)

Adjusting Saturation

Finally, the saturation of RGB images can be adjusted using tf.image.adjust_saturation.

In [7]:
Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.adjust_saturation(Images[0], saturation_factor = 1.5)
Aug_Names[0]+=' (Increased Quality by %50)'
Augmented_Images[1] = tf.image.adjust_saturation(Images[1], saturation_factor = .5)
Aug_Names[1]+=' (Decreased Quality by %50)'
# Image Show
_,_ = ImShow(Augmented_Images, Aug_Names, Title = False)
Cropping, Flipping and Rotating an Image

Central Croping

The central region of the image(s) can be cropped using tf.image.central_crop.

In [8]:
Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.central_crop(Images[0], central_fraction = 0.5)
Aug_Names[0]+=' (The central 50%)'
Augmented_Images[1] = tf.image.central_crop(Images[1], central_fraction = .1)
Aug_Names[1]+=' (The central 10%)'
# Image Show
_,_ = ImShow(Augmented_Images, Aug_Names, Title = False)

Flipping an Image

We can flip an image horizontally (left to right) using tf.image.flip_left_right, and flip an image vertically (upside down). using tf.image.flip_up_down.

In [9]:
Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.flip_left_right(Images[0])
Aug_Names[0]+=' (Filpped Left-to-Right)'
Augmented_Images[1] = tf.image.flip_up_down(Images[0])
Aug_Names[1] = Names[0] + ' (Filpped Up-to-Down)'
# Image Show
_,_ = ImShow(Augmented_Images, Aug_Names, Title = False)

Rotating an Image

To rotate an image counter-clockwise by multiples of 90 degrees, we can use tf.image.rot90. Here, k is the coefficients of that 90 degrees. For example, to achieve a rotation of 180 degrees, k can be chosen as 2.

In [10]:
Augmented_Images = []
Aug_Names = []
Augmented_Images.append(tf.image.rot90(Images[1], k = 1))
Aug_Names.append(Names[1] + ' (Rotated by 90 Degress)')
Augmented_Images.append(tf.image.rot90(Images[1], k = 2))
Aug_Names.append( Names[1] + ' (Rotated by 180 Degress)')
_,_ = ImShow(Augmented_Images, Aug_Names)
RGB, Grayscale and HSV

Grayscale and RGB

tf.image.rgb_to_grayscale converts images from RGB to Grayscale, and grayscale images can be converted to RGB using tf.image.grayscale_to_rgb.

In [11]:
from mpl_toolkits.axes_grid1.inset_locator import inset_axes

Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.squeeze(tf.image.rgb_to_grayscale(Images[1]))
Aug_Names[0]= Names[1] + ' (Grayscale)'
Augmented_Images[1] = tf.image.grayscale_to_rgb(tf.image.rgb_to_grayscale(Images[1]))
Aug_Names[1]+= Names[1] + ' (Grayscale to RGB)'

fig, ax = plt.subplots(1, 2 , figsize = (17, 6))
ax = ax.ravel()
font = FontProperties()
font.set_weight('bold')

for i in range(2):
    cs = ax[i].imshow(Augmented_Images[i], cmap='gray', vmin=0, vmax=255)
    _ = fig.colorbar(cs, cax = inset_axes(ax[i], width="2%", height="100%", loc='lower left',
                                          bbox_to_anchor=(1.05, 0., 1, 1), bbox_transform=ax[i].transAxes, borderpad=0))
    _ = ax[i].axis('tight')
    _ = ax[i].axis('off')
    _ = ax[i].set_aspect(aspect=1.0)
    _ = ax[i].set_title(Aug_Names[i], fontproperties=font, fontsize = 16)

RGB and HSV

Similarly, we can convert images from RGB to HSV back and fourth using tf.image.rgb_to_hsv and tf.image.hsv_to_rgb.

In [12]:
Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.rgb_to_hsv(Images[0])
Aug_Names[0] +=' (RGB-to-HSV)'
Augmented_Images[1] = tf.image.hsv_to_rgb(Augmented_Images[0])
Aug_Names[1] = Names[0] + ' (HSV-to-RGB)'
# Image Show
_,_ = ImShow([Augmented_Images[i]/255 for i in range(2)], Aug_Names, Title = False)

Image Gradients

A directional change in the intensity or color in an image is known as an image gradient which is one of the fundamental building blocks in image processing. To implement this in TensorFlow, we can use tf.image.image_gradients which returns image gradients (dy, dx) for each color.

In [13]:
Batch_Size = 1
Image_Height = Images[0].shape[0]
Image_Width = Images[0].shape[1]
Channels = 3
image = tf.reshape(tf.range(Image_Height * Image_Width * Channels, delta=1, dtype=tf.float32),
                   shape=(Batch_Size, Image_Height, Image_Width, Channels))
dx, dy = tf.image.image_gradients(image)

Image Standardization

tf.image.per_image_standardization linearly scales each image in image to have mean 0 and variance 1.

In [14]:
def ImgStd(Inp):
    '''This function standardizes image values.'''
    Out = np.round(Inp).astype(int)
    Out[Out<0] = 0
    Out[Out>255] = 255
    return Out

Augmented_Images = Images.copy()
Aug_Names = Names.copy()
Augmented_Images[0] = tf.image.rgb_to_hsv(Images[0])
Aug_Names[0] +=' (RGB-to-HSV)'
Augmented_Images[1] = tf.image.per_image_standardization(Augmented_Images[0])
Aug_Names[1] = Names[0] + ' (RGB-to-HSV Standardized)'

# Image Show
_,_ = ImShow(Augmented_Images, Aug_Names, Title = False)

Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Resizing Images

To resize images to a designated size with a specified method we can use tf.image.resize.

Resizing Methods Description
area Anti-aliased resampling with area interpolation.
bicubic Bicubic Interpolation
bilinear Bilinear Interpolation
gaussian Gaussian Filter
lanczos3 Lanczos Resampling with Radius 3
lanczos5 Lanczos Resampling with Radius 5
mitchellcubic Mitchell-Netravali Cubic Non-Interpolating Filter
nearest Nearest-Neighbor Interpolation

See the details here

In [15]:
Image = Images[1]
Name = Names[1]
Size_dec = (int(Image.shape[0]/3), int(Image.shape[1]/4))
Size_inc = (int(Image.shape[0]*2), int(Image.shape[1]*2))

Methods = {'area' : 'Area',
           'bicubic' : 'Bicubic Interpolation',
           'bilinear' : 'Bilinear Interpolation',
           'gaussian' : 'Gaussian Filter',
           'lanczos3' : 'Lanczos Resampling with Radius 3',
           'lanczos5' : 'Lanczos Resampling with Radius 5',
           'mitchellcubic' : 'Mitchell-Netravali Cubic Non-Interpolating Filter',
           'nearest' : 'Nearest-Neighbor Interpolation' }

for mtd in list(Methods.keys()):
    Augmented_Images = [];
    Augmented_Images.append(tf.image.resize(Image, size = Size_dec, method= 'area',
                                            preserve_aspect_ratio=False, antialias=True, name=None))
    Augmented_Images.append(tf.image.resize(Image, size = Size_inc, method= 'area',
                                            preserve_aspect_ratio=False, antialias=True, name=None))
    _,ax = ImShow([Augmented_Images[i]/255 for i in range(2)], ['Size Decreased', 'Size Increased'],
                 Title = 'Method: '+ Methods[mtd])
    _ = ax[0].text(x = 0, y = 0, s = 'Size: %i by %i' % (Augmented_Images[0].shape[0], Augmented_Images[0].shape[1]),
                  size=13, color = 'Navy', bbox=dict(boxstyle="square", ec='Navy',fc='LightSkyBlue'))
    _ = ax[1].text(x = 0, y = 0, s = 'Size: %i by %i' % (Augmented_Images[1].shape[0], Augmented_Images[1].shape[1]),
                  size=13, color = 'Navy', bbox=dict(boxstyle="square", ec='Navy',fc='LightSkyBlue'))

for i in range(len(ax)):
    _ = ax[i].set_aspect(aspect=1)
    _ = ax[i].axis('off')
del i
Random Image Adjustments
Syntax Description
tf.image.random_brightness Adjust the brightness of images by a random factor.
tf.image.random_contrast Adjust the contrast of an image or images by a random factor.
tf.image.random_crop Randomly crops a tensor to a given size.
tf.image.random_flip_left_right Randomly flip an image horizontally (left to right).
tf.image.random_flip_up_down Randomly flips an image vertically (upside down).
tf.image.random_hue Adjust the hue of RGB images by a random factor.
tf.image.random_jpeg_quality Randomly changes jpeg encoding quality for inducing jpeg noise.
tf.image.random_saturation Adjust the saturation of RGB images by a random factor.
In [16]:
import numpy as np
Augmented_Images = []
Aug_Names = []

Image = Images[1]
Name = Names[1]

## Random Brightness
Augmented_Images.append(tf.image.random_brightness(Image, max_delta = 0.3, seed=1234))
Aug_Names.append(Name + ' (Random Brightness)')

## Random Contrast
Augmented_Images.append(tf.image.random_contrast(Image, lower= 0.1, upper=0.5, seed=1234))
Aug_Names.append(Name + ' (Random Contrast)')

## Random Crop
Augmented_Images.append(tf.image.random_crop(Image, size=[int(Image.shape[0]/2),
                                                              int(Image.shape[1]/2), 3]))
Aug_Names.append(Name + ' (Random Crop)')

## Random Flip Left-to-Right
Augmented_Images.append(tf.image.random_flip_left_right(Image, seed=1234))
Aug_Names.append(Name + ' (Random Flip Left-to-Right)')

## Random Flip Up-to-Down
Augmented_Images.append(tf.image.random_flip_up_down(Image, seed=1234))
Aug_Names.append(Name + ' (Random Flip Up-to-Down)')

## Random Hue
Augmented_Images.append(tf.image.random_hue(Image, max_delta = 0.3, seed=1234))
Aug_Names.append(Name + ' (Random Hue)')

## Random JPEG Quality
Augmented_Images.append(tf.image.random_jpeg_quality(Image, min_jpeg_quality = 15, max_jpeg_quality = 50, seed=1234))
Aug_Names.append(Name + ' (Random JPEG Quality)')

## Random Saturation
Augmented_Images.append(tf.image.random_saturation(Image, lower =  3, upper = 8, seed=1234))
Aug_Names.append(Name + ' (Random Saturation)')

## 

N = len(Augmented_Images)
rows = int(np.ceil(N/2))
fig, ax = plt.subplots(rows, 2 , figsize = (17, 5*rows))
ax = ax.ravel()
font = FontProperties()
font.set_weight('bold')

for i in range(N):
    _ = ax[i].imshow(Augmented_Images[i])
    _ = ax[i].axis('off')
    _ = ax[i].set_aspect(aspect=1)
    _ = ax[i].set_title(Aug_Names[i], fontproperties=font, fontsize = 16)
    
del N, rows, i

References

  1. TensorFlow Image
  2. Brightness Wikipedia page
  3. Contrast (vision) Wikipedia page)
  4. Hue Wikipedia page
  5. Image gradient Wikipedia page
  6. Interpolation Algorithms in PixInsight
  7. Bicubic Interpolation Wikipedia page
  8. Bilinear Interpolation Wikipedia page
  9. Gaussian Filter Wikipedia page
  10. Lanczos Resampling Wikipedia page
  11. Nearest-Neighbor Interpolation Wikipedia page